/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package edu.boun.g4.coursity.controller;

import java.io.File;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Pattern;

import com.google.common.base.Splitter;
import com.google.common.collect.Lists;
import com.google.common.io.Closeables;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.model.AbstractDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericBooleanPrefDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;
import org.apache.mahout.common.iterator.FileLineIterator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.google.common.base.Preconditions;

import edu.boun.g4.coursity.dto.RatingDto;

/**
 * <p>
 * A {@link DataModel} backed by a delimited file. This class expects a file where each line
 * contains a user ID, followed by item ID, followed by optional preference value, followed by
 * optional timestamp. Commas or tabs delimit fields:
 * </p>
 *
 * <p>{@code userID,itemID[,preference[,timestamp]]}</p>
 *
 * <p>
 * Preference value is optional to accommodate applications that have no notion of a
 * preference value (that is, the user simply expresses a
 * preference for an item, but no degree of preference).
 * </p>
 *
 * <p>
 * The preference value is assumed to be parseable as a {@code double}. The user IDs and item IDs are
 * read parsed as {@code long}s. The timestamp, if present, is assumed to be parseable as a
 * {@code long}, though this can be overridden via {@link #readTimestampFromString(String)}.
 * The preference value may be empty, to indicate "no preference value", but cannot be empty. That is,
 * this is legal:
 * </p>
 *
 * <p>{@code 123,456,,129050099059}</p>
 *
 * <p>But this isn't:</p>
 *
 * <p>{@code 123,456,129050099059}</p>
 *
 * <p>
 * It is also acceptable for the lines to contain additional fields. Fields beyond the third will be ignored.
 * An empty line, or one that begins with '#' will be ignored as a comment.
 * </p>
 *
 * <p>
 * This class will reload data from the data file when {@link #refresh(Collection)} is called, unless the file
 * has been reloaded very recently already.
 * </p>
 *
 * <p>
 * This class will also look for update "delta" files in the same directory, with file names that start the
 * same way (up to the first period). These files have the same format, and provide updated data that
 * supersedes what is in the main data file. This is a mechanism that allows an application to push updates to
 *  without re-copying the entire data file.
 * </p>
 *
 * <p>
 * One small format difference exists. Update files must also be able to express deletes.
 * This is done by ending with a blank preference value, as in "123,456,".
 * </p>
 *
 * <p>
 * Note that it's all-or-nothing -- all of the items in the file must express no preference, or the all must.
 * These cannot be mixed. Put another way there will always be the same number of delimiters on every line of
 * the file!
 * </p>
 *
 * <p>
 * This class is not intended for use with very large amounts of data (over, say, tens of millions of rows).
 * For that, a JDBC-backed {@link DataModel} and a database are more appropriate.
 * </p>
 *
 * <p>
 * It is possible and likely useful to subclass this class and customize its behavior to accommodate
 * application-specific needs and input formats. See {@link #processLine(String, FastByIDMap, FastByIDMap, boolean)} and
 * {@link #processLineWithoutID(String, FastByIDMap, FastByIDMap)}
 */
public class CourseDataModel extends AbstractDataModel {

  private static final Logger log = LoggerFactory.getLogger(CourseDataModel.class);
  public static final long DEFAULT_MIN_RELOAD_INTERVAL_MS = 60 * 1000L; // 1 minute?

  
  private final char delimiter;
  private final Splitter delimiterPattern;
  private final boolean hasPrefValues = true;
  private DataModel delegate;
  private final ReentrantLock reloadLock;
  private final boolean transpose;
  private final long minReloadIntervalMS;  
  
  private static final char COMMENT_CHAR = '#';

  private final List<RatingDto> ratingsData;

  /**
   * @param transpose
   *          transposes user IDs and item IDs -- convenient for 'flipping' the data model this way
   * @param minReloadIntervalMS
   *  the minimum interval in milliseconds after which a full reload of the original datafile is done
   *  when refresh() is called
   * @see #CourseDataModel(File)
   */
  public CourseDataModel(List<RatingDto> ratings) throws Exception {    

	this.ratingsData = ratings;      
    
    delimiter = ',';
    delimiterPattern = Splitter.on(delimiter);
    
    this.reloadLock = new ReentrantLock();
    this.transpose = false;
    this.minReloadIntervalMS = DEFAULT_MIN_RELOAD_INTERVAL_MS;
           
    reload();
  } 

  public char getDelimiter() {
    return delimiter;
  }

  protected void reload() {
    if (reloadLock.tryLock()) {
      try {
        delegate = buildModel();
      } catch (Exception ioe) {
        log.warn("Exception while reloading", ioe);
      } finally {
        reloadLock.unlock();
      }
    }
  }

  protected DataModel buildModel() throws Exception {
   

    boolean loadFreshData = delegate == null;
    FastByIDMap<FastByIDMap<Long>> timestamps = new FastByIDMap<FastByIDMap<Long>>();

    if (hasPrefValues) {

      if (loadFreshData) {

    	  FastByIDMap<Collection<Preference>> data = new FastByIDMap<Collection<Preference>>();    	    
    	  
    	  processData(this.ratingsData, data, timestamps, false);    	  
    	  
    	  return new GenericDataModel(GenericDataModel.toDataMap(data, true), timestamps);

      } 
      else {

        FastByIDMap<PreferenceArray> rawData = ((GenericDataModel) delegate).getRawUserData();

        processData(this.ratingsData, rawData, timestamps, true);    	  
        
        return new GenericDataModel(rawData, timestamps);

      }

    } else {

    	
      if (loadFreshData) {

        FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>();
        
        /*
        FileLineIterator iterator = new FileLineIterator(dataFile, false);
        processFileWithoutID(iterator, data, timestamps);

        for (File updateFile : findUpdateFilesAfter(newLastModified)) {
          processFileWithoutID(new FileLineIterator(updateFile, false), data, timestamps);
        }
        */

        return new GenericBooleanPrefDataModel(data, timestamps);

      } else {
		
        FastByIDMap<FastIDSet> rawData = ((GenericBooleanPrefDataModel) delegate).getRawUserData();
/*
        for (File updateFile : findUpdateFilesAfter(Math.max(oldLastUpdateFileModifieid, newLastModified))) {
          processFileWithoutID(new FileLineIterator(updateFile, false), rawData, timestamps);
        }
*/
        return new GenericBooleanPrefDataModel(rawData, timestamps);    

      }   

    }
  }

 

  protected void processData(List<RatingDto> ratings,
                             FastByIDMap<?> data,
                             FastByIDMap<FastByIDMap<Long>> timestamps,
                             boolean fromPriorData) {
    log.info("Reading data...");
    int count = 0;
    
    for(Iterator<RatingDto> i = ratings.iterator(); i.hasNext(); ) {
    	RatingDto item = i.next();
    	
    	if (item != null)
    	{
    		String strLine = item.getUserId() + "," + item.getCourseId() + "," + item.getRatingScore()+ "," + item.getId();
    		
    		if (!strLine.isEmpty())
    		{
    			processLine(strLine, data, timestamps, fromPriorData);
    			if (++count % 1000000 == 0) {
    		          log.info("Processed {} lines", count);
    		        }
    		}
    	}
    }   
   
    log.info("Read lines: {}", count);
  }

  
  
  /**
   * <p>
   * Reads one line from the input file and adds the data to a {@link FastByIDMap} data structure which maps user IDs
   * to preferences. This assumes that each line of the input file corresponds to one preference. After
   * reading a line and determining which user and item the preference pertains to, the method should look to
   * see if the data contains a mapping for the user ID already, and if not, add an empty data structure of preferences
   * as appropriate to the data.
   * </p>
   *
   * <p>
   * Note that if the line is empty or begins with '#' it will be ignored as a comment.
   * </p>
   *
   * @param line
   *          line from input data file
   * @param data
   *          all data read so far, as a mapping from user IDs to preferences
   * @param fromPriorData an implementation detail -- if true, data will map IDs to
   *  {@link PreferenceArray} since the framework is attempting to read and update raw
   *  data that is already in memory. Otherwise it maps to {@link Collection}s of
   *  {@link Preference}s, since it's reading fresh data. Subclasses must be prepared
   *  to handle this wrinkle.
   */
  protected void processLine(String line,
                             FastByIDMap<?> data, 
                             FastByIDMap<FastByIDMap<Long>> timestamps,
                             boolean fromPriorData) {

    // Ignore empty lines and comments
    if (line.isEmpty() || line.charAt(0) == COMMENT_CHAR) {
      return;
    }

    Iterator<String> tokens = delimiterPattern.split(line).iterator();
    String userIDString = tokens.next();
    String itemIDString = tokens.next();
    String preferenceValueString = tokens.next();
    boolean hasTimestamp = tokens.hasNext();
    String timestampString = hasTimestamp ? tokens.next() : null;

    long userID = readUserIDFromString(userIDString);
    long itemID = readItemIDFromString(itemIDString);

    if (transpose) {
      long tmp = userID;
      userID = itemID;
      itemID = tmp;
    }

    // This is kind of gross but need to handle two types of storage
    Object maybePrefs = data.get(userID);
    if (fromPriorData) {
      // Data are PreferenceArray

      PreferenceArray prefs = (PreferenceArray) maybePrefs;
      if (!hasTimestamp && preferenceValueString.isEmpty()) {
        // Then line is of form "userID,itemID,", meaning remove
        if (prefs != null) {
          boolean exists = false;
          int length = prefs.length();
          for (int i = 0; i < length; i++) {
            if (prefs.getItemID(i) == itemID) {
              exists = true;
              break;
            }
          }
          if (exists) {
            if (length == 1) {
              data.remove(userID);
            } else {
              PreferenceArray newPrefs = new GenericUserPreferenceArray(length - 1);
              for (int i = 0, j = 0; i < length; i++, j++) {
                if (prefs.getItemID(i) == itemID) {
                  j--;
                } else {
                  newPrefs.set(j, prefs.get(i));
                }
              }
            }
          }
        }

        removeTimestamp(userID, itemID, timestamps);

      } else {

        float preferenceValue = Float.parseFloat(preferenceValueString);

        boolean exists = false;
        if (prefs != null) {
          for (int i = 0; i < prefs.length(); i++) {
            if (prefs.getItemID(i) == itemID) {
              exists = true;
              prefs.setValue(i, preferenceValue);
              break;
            }
          }
        }

        if (!exists) {
          if (prefs == null) {
            prefs = new GenericUserPreferenceArray(1);
          } else {
            PreferenceArray newPrefs = new GenericUserPreferenceArray(prefs.length() + 1);
            for (int i = 0, j = 1; i < prefs.length(); i++, j++) {
              newPrefs.set(j, prefs.get(i));
            }
            prefs = newPrefs;
          }
          prefs.setUserID(0, userID);
          prefs.setItemID(0, itemID);
          prefs.setValue(0, preferenceValue);
          ((FastByIDMap<PreferenceArray>) data).put(userID, prefs);          
        }
      }

      addTimestamp(userID, itemID, timestampString, timestamps);

    } else {
      // Data are Collection<Preference>

      Collection<Preference> prefs = (Collection<Preference>) maybePrefs;

      if (!hasTimestamp && preferenceValueString.isEmpty()) {
        // Then line is of form "userID,itemID,", meaning remove
        if (prefs != null) {
          // remove pref
          Iterator<Preference> prefsIterator = prefs.iterator();
          while (prefsIterator.hasNext()) {
            Preference pref = prefsIterator.next();
            if (pref.getItemID() == itemID) {
              prefsIterator.remove();
              break;
            }
          }
        }

        removeTimestamp(userID, itemID, timestamps);
        
      } else {

        float preferenceValue = Float.parseFloat(preferenceValueString);

        boolean exists = false;
        if (prefs != null) {
          for (Preference pref : prefs) {
            if (pref.getItemID() == itemID) {
              exists = true;
              pref.setValue(preferenceValue);
              break;
            }
          }
        }

        if (!exists) {
          if (prefs == null) {
            prefs = Lists.newArrayListWithCapacity(2);
            ((FastByIDMap<Collection<Preference>>) data).put(userID, prefs);
          }
          prefs.add(new GenericPreference(userID, itemID, preferenceValue));
        }

        addTimestamp(userID, itemID, timestampString, timestamps);

      }

    }
  }
 

  private void addTimestamp(long userID,
                            long itemID,
                            String timestampString,
                            FastByIDMap<FastByIDMap<Long>> timestamps) {
    if (timestampString != null) {
      FastByIDMap<Long> itemTimestamps = timestamps.get(userID);
      if (itemTimestamps == null) {
        itemTimestamps = new FastByIDMap<Long>();
        timestamps.put(userID, itemTimestamps);
      }
      long timestamp = readTimestampFromString(timestampString);
      itemTimestamps.put(itemID, timestamp);
    }
  }

  private static void removeTimestamp(long userID,
                                      long itemID,
                                      FastByIDMap<FastByIDMap<Long>> timestamps) {
    FastByIDMap<Long> itemTimestamps = timestamps.get(userID);
    if (itemTimestamps != null) {
      itemTimestamps.remove(itemID);
    }
  }

  /**
   * Subclasses may wish to override this if ID values in the file are not numeric. This provides a hook by
   * which subclasses can inject an {@link org.apache.mahout.cf.taste.model.IDMigrator} to perform
   * translation.
   */
  protected long readUserIDFromString(String value) {
    return Long.parseLong(value);
  }

  /**
   * Subclasses may wish to override this if ID values in the file are not numeric. This provides a hook by
   * which subclasses can inject an {@link org.apache.mahout.cf.taste.model.IDMigrator} to perform
   * translation.
   */
  protected long readItemIDFromString(String value) {
    return Long.parseLong(value);
  }

  /**
   * Subclasses may wish to override this to change how time values in the input file are parsed.
   * By default they are expected to be numeric, expressing a time as milliseconds since the epoch.
   */
  protected long readTimestampFromString(String value) {
    return Long.parseLong(value);
  }

  @Override
  public LongPrimitiveIterator getUserIDs() throws TasteException {
    return delegate.getUserIDs();
  }

  @Override
  public PreferenceArray getPreferencesFromUser(long userID) throws TasteException {
    return delegate.getPreferencesFromUser(userID);
  }

  @Override
  public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
    return delegate.getItemIDsFromUser(userID);
  }

  @Override
  public LongPrimitiveIterator getItemIDs() throws TasteException {
    return delegate.getItemIDs();
  }

  @Override
  public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
    return delegate.getPreferencesForItem(itemID);
  }

  @Override
  public Float getPreferenceValue(long userID, long itemID) throws TasteException {
    return delegate.getPreferenceValue(userID, itemID);
  }

  @Override
  public Long getPreferenceTime(long userID, long itemID) throws TasteException {
    return delegate.getPreferenceTime(userID, itemID);
  }

  @Override
  public int getNumItems() throws TasteException {
    return delegate.getNumItems();
  }

  @Override
  public int getNumUsers() throws TasteException {
    return delegate.getNumUsers();
  }

  @Override
  public int getNumUsersWithPreferenceFor(long itemID) throws TasteException {
    return delegate.getNumUsersWithPreferenceFor(itemID);
  }

  @Override
  public int getNumUsersWithPreferenceFor(long itemID1, long itemID2) throws TasteException {
    return delegate.getNumUsersWithPreferenceFor(itemID1, itemID2);
  }

  /**
   * Note that this method only updates the in-memory preference data that this
   * maintains; it does not modify any data on disk. Therefore any updates from this method are only
   * temporary, and lost when data is reloaded from a file. This method should also be considered relatively
   * slow.
   */
  @Override
  public void setPreference(long userID, long itemID, float value) throws TasteException {
    delegate.setPreference(userID, itemID, value);
  }

  /** See the warning at {@link #setPreference(long, long, float)}. */
  @Override
  public void removePreference(long userID, long itemID) throws TasteException {
    delegate.removePreference(userID, itemID);
  }

  @Override
  public void refresh(Collection<Refreshable> alreadyRefreshed) {    
      reload();    
  }

  @Override
  public boolean hasPreferenceValues() {
    return delegate.hasPreferenceValues();
  }

  @Override
  public float getMaxPreference() {
    return delegate.getMaxPreference();
  }

  @Override
  public float getMinPreference() {
    return delegate.getMinPreference();
  }

  @Override
  public String toString() {
    return "CourseDataModel[ratings]";
  }

}
